import requests
from fake_useragent import UserAgent
from lxml import etree
import aiohttp
import asyncio

# for i in range(1, 31):
#     if i == 1:
#         url = 'https://www.minebbs.com/forums/news/'
#     else:
#         url = f'https://www.minebbs.com/forums/news/page-{i}'
headers = {
    'User-Agent': UserAgent().random,
}


#
#     response = requests.get(url, headers=headers)
#     tree = etree.HTML(response.text)
#     line_tree = tree.xpath('//div[@class="structItemContainer-group js-threadList"]/div')
#     for line in line_tree:
#         line_data = line.xpath('.//div[@class="structItem-title"]/a[2]')[0]
#         title = line_data.xpath('./text()')[0].strip()
#         url = line_data.xpath('./@href')[0]
#         detail_url = 'https://www.minebbs.com' + url
#         print(title, detail_url)

async def get_title(session, url):
    async with session.get(url, headers=headers) as response:
        html_str = await response.text()
        tree = etree.HTML(html_str)
        line_tree = tree.xpath('//div[@class="structItemContainer-group js-threadList"]/div')
        next = tree.xpath('//a[@class="pageNav-jump pageNav-jump--next"]')
        for line in line_tree:
            line_data = line.xpath('.//div[@class="structItem-title"]/a[2]')
            if not line_data:
                line_data = line.xpath('.//div[@class="structItem-title"]/a')
            title = line_data[0].xpath('./text()')[0].strip()
            url = line_data[0].xpath('./@href')[0]
            detail_url = 'https://www.minebbs.com' + url
            print(title, detail_url)
        if next:
            next_url = 'https://www.minebbs.com' + next[0].xpath('./@href')[0]
            print(next_url)
            await get_title(session, next_url)
        else:
            return 'over！！！！！！！！！！'


async def main():
    async with aiohttp.ClientSession() as session:
        await asyncio.gather(get_title(session, 'https://www.minebbs.com/forums/news/'))


asyncio.run(main())
